{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import inspect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 查看hdf文件是否已经生成\n",
    "def check_hdf(func, dtype='yyzz'):\n",
    "    file = func.__name__.split('_', maxsplit=1)[1]\n",
    "    path = f'../output/{file}'\n",
    "    if os.path.exists(path):\n",
    "        df = pd.read_hdf(path, dtype)\n",
    "        print('>>> read   ', path, dtype)\n",
    "        return df\n",
    "    else:\n",
    "        print('>>> generate   ', path, dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_memory(data):\n",
    "    start_memory = data.memory_usage().sum() / 1024**2 \n",
    "    print(\"Memory usage of properties dataframe is :\",start_memory,\" MB\")\n",
    "    NAlist = [] # Keeps track of columns that have missing values filled in. \n",
    "    \n",
    "    for col in data.columns:\n",
    "        if ('int' in data[col].dtype.name) or ('float' in data[col].dtype.name):  # Exclude strings\n",
    "            try:\n",
    "                # Print current column type\n",
    "                print(\"******************************\")\n",
    "                print(\"Column: \",col)\n",
    "                print(\"dtype before: \",data[col].dtype)\n",
    "\n",
    "                # make variables for Int, max and min\n",
    "                IsInt = False\n",
    "                value_max = data[col].max()\n",
    "                value_min = data[col].min()\n",
    "\n",
    "                # Integer does not support NA, therefore, NA needs to be filled\n",
    "                if not np.isfinite(data[col]).all(): \n",
    "                    NAlist.append(col)\n",
    "                    data[col].fillna(value_min-1,inplace=True)  \n",
    "\n",
    "                # test if column can be converted to an integer\n",
    "                asint = data[col].fillna(0).astype(np.int64)\n",
    "                result = (data[col] - asint)\n",
    "                result = result.sum()\n",
    "                if result > -0.01 and result < 0.01:\n",
    "                    IsInt = True\n",
    "\n",
    "\n",
    "                # Make Integer/unsigned Integer datatypes\n",
    "                if IsInt:\n",
    "                    if value_min >= 0:\n",
    "                        if value_max < 255:\n",
    "                            data[col] = data[col].astype(np.uint8)\n",
    "                        elif value_max < 65535:\n",
    "                            data[col] = data[col].astype(np.uint16)\n",
    "                        elif value_max < 4294967295:\n",
    "                            data[col] = data[col].astype(np.uint32)\n",
    "                        else:\n",
    "                            data[col] = data[col].astype(np.uint64)\n",
    "                    else:\n",
    "                        if value_min > np.iinfo(np.int8).min and value_max < np.iinfo(np.int8).max:\n",
    "                            data[col] = data[col].astype(np.int8)\n",
    "                        elif value_min > np.iinfo(np.int16).min and value_max < np.iinfo(np.int16).max:\n",
    "                            data[col] = data[col].astype(np.int16)\n",
    "                        elif value_min > np.iinfo(np.int32).min and value_max < np.iinfo(np.int32).max:\n",
    "                            data[col] = data[col].astype(np.int32)\n",
    "                        elif value_min > np.iinfo(np.int64).min and value_max < np.iinfo(np.int64).max:\n",
    "                            data[col] = data[col].astype(np.int64)    \n",
    "\n",
    "                # Make float datatypes 32 bit\n",
    "                else:\n",
    "                    data[col] = data[col].astype(np.float32)\n",
    "\n",
    "                # Print new column type\n",
    "                print(\"dtype after: \",data[col].dtype)\n",
    "                print(\"******************************\")\n",
    "            except:\n",
    "                print(\"dtype after: Failed\")\n",
    "        else:\n",
    "            print(\"dtype remain: \",data[col].dtype)\n",
    "    \n",
    "    # Print final result\n",
    "    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n",
    "    end_memory = data.memory_usage().sum() / 1024**2 \n",
    "    print(\"Memory usage is: \",end_memory,\" MB\")\n",
    "    print(\"This is \",100*start_memory/end_memory,\"% of the initial size\")\n",
    "    print(\"Missing Value list\", NAlist)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 检查提交文件是否正确\n",
    "def check_subfile(df, name):\n",
    "    label = pd.read_csv('../data/Antai_AE_round2_test_20190813.zip')\n",
    "    df = df.merge(label[['buyer_admin_id']].drop_duplicates(), how='right', on=['buyer_admin_id']).fillna(0)\n",
    "    \n",
    "    if len(df) != 9844:\n",
    "        raise ValueError('sub file lenght is not correct')\n",
    "    \n",
    "    if len(df.columns) != 31:\n",
    "        raise ValueError('sub file column is not correct')\n",
    "        \n",
    "    if df.columns[0] != 'buyer_admin_id':\n",
    "        raise ValueError('sub file columns dont contains buyer_country_id')\n",
    "    df.to_csv('../submit/' + name, index=False, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# top30sku，行列转换\n",
    "def submit_transform(df, name=None):\n",
    "    # 过滤掉不在训练集的商品\n",
    "    train_item = get_hdf('train')['item_id'].drop_duplicates()\n",
    "    df = df[df['item_id'].isin(train_item)]\n",
    "    df['irank'] = df.groupby(['buyer_admin_id']).cumcount()+1\n",
    "    \n",
    "    df = df[df['irank']<=30].set_index(['buyer_admin_id', 'irank']).unstack(-1)\n",
    "    df = df.fillna(11717821).astype(int)\n",
    "    df.columns = [int(col) if isinstance(col, float) else col for col in df.columns.droplevel(0)]\n",
    "    df = df.reset_index()\n",
    "    if name is None:\n",
    "        name = datetime.datetime.today().strftime('%m-%d') + '.csv'\n",
    "    check_subfile(df, name)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_hdf():\n",
    "    train = pd.read_csv('../data/Antai_AE_round2_train_20190813.zip')\n",
    "    test = pd.read_csv('../data/Antai_AE_round2_test_20190813.zip')\n",
    "    item = pd.read_csv('../data/Antai_AE_round2_item_attr_20190813.zip')\n",
    "\n",
    "    df = pd.concat([train.assign(is_train=1), test.assign(is_train=0)])\n",
    "    df.sort_values(by=['country_id', 'buyer_admin_id', 'irank'], ascending=[1, 1, 0], inplace=True)\n",
    "    \n",
    "    # 创建时间信息列\n",
    "    df['log_time'] = pd.to_datetime(df['log_time'])\n",
    "    df['time_rank'] = df.groupby(['buyer_admin_id'])['log_time'].rank(ascending=False, method='dense')\n",
    "    df['log_time'] = df['log_time'].astype(str)\n",
    "    df['date'] = df['log_time'].str[:10]\n",
    "    df['day'] = df['log_time'].str[8:10].astype(int)\n",
    "    \n",
    "    # 用户每条记录与首末(irank2为末)记录时间点\n",
    "    second = df[df['irank']>1].groupby(['buyer_admin_id']).agg(first_second_diff=('second','min'), last_second_diff=('second','max')).reset_index()\n",
    "    df = pd.merge(df, second, on=['buyer_admin_id'], how='left')\n",
    "    df['first_second_diff'] = df['second'] - df['first_second_diff']\n",
    "    df['last_second_diff'] = df['second'] - df['last_second_diff']\n",
    "    df.loc[df['irank']==1, 'first_second_diff'] = np.nan\n",
    "    df.loc[df['irank']==1, 'last_second_diff'] = np.nan\n",
    "    \n",
    "    df = df.sort_values(by=['country_id', 'buyer_admin_id', 'irank'], ascending=[1, 1, 1]).reset_index(drop=True)\n",
    "    df = reduce_memory(df)\n",
    "    df = pd.merge(df, item, how='left', on='item_id')\n",
    "    memory = df.memory_usage().sum() / 1024**2 \n",
    "    print('After memory usage of properties dataframe is :', memory, \" MB\")\n",
    "    \n",
    "    # 处理irank=1但是buy_flag为0的数据\n",
    "    df = df.sort_values(by=['buyer_admin_id', 'log_time', 'buy_flag'], ascending=[1, 0, 0])\n",
    "    df['irank'] = df.groupby(['buyer_admin_id']).cumcount() + 1\n",
    "    df.loc[df['is_train']==0, 'irank'] = df.loc[df['is_train']==0, 'irank'] + 1\n",
    "    df['second_irankn'] = df['second']\n",
    "    df.loc[df['irank']==1, 'second_irankn'] = np.nan\n",
    "    df['dense_rank'] = df.groupby(['buyer_admin_id'])['second_irankn'].rank(method='dense', ascending=False)\n",
    "    df = df.reset_index(drop=True)\n",
    "    \n",
    "    # 生成hdf文件\n",
    "    df[df['is_train']==1][:30000000].to_hdf('../data/base', 'train-half1')\n",
    "    df[df['is_train']==1][30000000:].to_hdf('../data/base', 'train-half2')\n",
    "    df[df['is_train']==0].to_hdf('../data/base', 'test')\n",
    "    df[df['country_id']=='xx'].to_hdf('../data/base', 'xx')\n",
    "    df[df['country_id']=='yy'].to_hdf('../data/base', 'yy')\n",
    "    df[df['country_id']=='zz'].to_hdf('../data/base', 'zz')\n",
    "    df[df['buy_flag']==1].to_hdf('../data/base', 'buy')\n",
    "    df[df['irank']==1].to_hdf('../data/base', 'irank1')\n",
    "    df[df['irank']==2].to_hdf('../data/base', 'irank2')\n",
    "    df[df['irank']==3].to_hdf('../data/base', 'irank3')\n",
    "    df[df['irank']==4].to_hdf('../data/base', 'irank4')\n",
    "    df[df['irank']==5].to_hdf('../data/base', 'irank5')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取热销商品\n",
    "def get_hot(dtype='all'):\n",
    "    # 提取整体数据\n",
    "    df = get_hdf(dtype='yyzz')\n",
    "    \n",
    "    # 每日热销商品与品类热销(top30)\n",
    "    if dtype in ['date', 'cate_id', 'store_id']:\n",
    "        # 计算某天/某品类下，top30热销商品\n",
    "        hot = df.groupby([dtype, 'item_id'])['buyer_admin_id'].nunique().to_frame('count').reset_index().sort_values([dtype, 'count'],ascending=[1,0])\n",
    "        hot = hot.groupby([dtype]).head(60).drop('count', 1)\n",
    "        hot['irank'] = hot.groupby(dtype).cumcount() + 30\n",
    "        hot.reset_index(drop=True).to_hdf('../data/hot.h5', dtype)\n",
    "        return hot\n",
    "    \n",
    "    # 整体热销商品(top30)\n",
    "    elif dtype=='all':\n",
    "        buyer_admin_id = df['buyer_admin_id'].unique()\n",
    "        item_id = df.groupby(['item_id'])['buyer_admin_id'].nunique().sort_values(ascending=False).head(60).index.tolist()\n",
    "        hot = pd.DataFrame(index = pd.MultiIndex.from_product([buyer_admin_id, item_id], names=['buyer_admin_id', 'item_id'])).reset_index()\n",
    "        hot['irank'] = hot.groupby(['buyer_admin_id']).cumcount() + 30\n",
    "        hot.reset_index(drop=True).to_hdf('../data/hot.h5', 'all')\n",
    "        return hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_hdf(dtype='all', data_type='base', if_filter_label=False, if_lastday=False, if_drop_duplicates=False, if_debug=False, is_train=None):\n",
    "    \"\"\"\n",
    "    data_type: base 原始数据文件 \n",
    "    data_type: slide 向前滑一次购买记录\n",
    "    data_type: slide_recall 冷启动划窗购买记录\n",
    "    \n",
    "        dtype='train'，训练集\n",
    "        dtype='test'，测试集\n",
    "        dtype='all'，训练集+测试集\n",
    "        dtype='yyzz'，yy国和zz国数据\n",
    "        dtype='buy'， buy_flag=1的所有数据 \n",
    "        dtype='irank1'， irank=1的所有数据 \n",
    "        dtype='irank2'， irank=2的所有数据 \n",
    "    \n",
    "    if_filter_label : 是否过滤label\n",
    "    if_lastday ： 是否只取最后一天数据()\n",
    "    if_drop_duplicates : 是否过滤重复数据(按秒去重)\n",
    "    if_debug : debug取前10000行\n",
    "    is_train : 是否只取训练数据\n",
    "    \"\"\"\n",
    "    # 基础数据文件\n",
    "    path = '../data/' + data_type\n",
    "    if dtype == 'all':\n",
    "        df = pd.concat([pd.read_hdf(path, 'train-half1'),\n",
    "                        pd.read_hdf(path, 'train-half2'),\n",
    "                        pd.read_hdf(path, 'test')])\n",
    "    elif dtype == 'train':\n",
    "        df = pd.concat([pd.read_hdf(path, 'train-half1'),\n",
    "                        pd.read_hdf(path, 'train-half2')])\n",
    "    elif dtype == 'yyzz':\n",
    "        df = pd.concat([pd.read_hdf(path, 'yy'),\n",
    "                        pd.read_hdf(path, 'zz')])\n",
    "    else:\n",
    "        df = pd.read_hdf(path, dtype)\n",
    "\n",
    "    # 过滤irank=1，在做特征时使用，防止加入label信息\n",
    "    if if_filter_label:\n",
    "        df = df[df['irank']!=1]\n",
    "\n",
    "    # 用户最后一天行为的数据, 做特征时，需要if_filter_label=True，防止把irank1的日期作为最后一天，泄露label信息\n",
    "    if if_lastday:\n",
    "        last_day = df.groupby(['buyer_admin_id'])['day'].max().to_frame('last_day').reset_index()\n",
    "        df = df.merge(last_day, on=['buyer_admin_id'], how='left')\n",
    "        df = df[df['day']==df['last_day']]\n",
    "\n",
    "    # 过滤数据大量，同一数据重复的数据\n",
    "    if if_drop_duplicates:\n",
    "        df = df.drop_duplicates(subset=['buyer_admin_id', 'item_id', 'second'], keep='first').reset_index(drop=True)\n",
    "        df['irank_dedup'] = df.groupby(['buyer_admin_id']).cumcount()\n",
    "    \n",
    "    if is_train is not None:\n",
    "        if is_train:\n",
    "            df = df[df['is_train']==1]\n",
    "        else:\n",
    "            df = df[df['is_train']==0]\n",
    "            \n",
    "    if if_debug:\n",
    "        df = df[:100000]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_user(dtype, data_type='base'):\n",
    "    \"\"\"用户分群：\n",
    "    all：全用户 624804\n",
    "        train + test = all\n",
    "        > train：训练集用户 614960\n",
    "        \n",
    "            按是否有历史购买记录划分：train_buy + non_train_buy = train \n",
    "            >> train_buy : 历史有购买记录用户（irank2 > 1 & buy_flag = 1）558565\n",
    "            >> non_train_buy : 历史无购买记录用户（irank2 > 1 & buy_flag = 0） 56395\n",
    "        \n",
    "            按是否冷启动划分：rebuy_user + cold_user = train\n",
    "            >> rebuy_user : irank1为历史复购 418821  --当前rank模型训练用户样本\n",
    "            >> cold_user :  irank1为首次购买 196139  --当前recall模型训练用户样本\n",
    "        \n",
    "                cold_buy_user + non_cold_buy_user = cold_user\n",
    "                >>> cold_buy_user : irank1为首次购买，但历史有购买记录 139746\n",
    "                >>> non_cold_buy_user : irank1为首次购买，但是历史无购买记录 56393\n",
    "        \n",
    "        > test：测试集用户 9844\n",
    "            按是否有历史购买记录划分：test_buy + non_test_buy = test \n",
    "            >> test_buy : 历史有购买记录用户（irank2 > 1 & buy_flag = 1）8947\n",
    "            >> non_test_buy : 历史无购买记录用户（irank2 > 1 & buy_flag = 0）897\n",
    "        \n",
    "        recall : cold + test 205983\n",
    "    \"\"\"\n",
    "    path = '../data/user_' + data_type\n",
    "    if os.path.exists(path):\n",
    "        user = pd.read_hdf(path, dtype).reset_index(drop=True)\n",
    "        return user\n",
    "    else:\n",
    "        \n",
    "        df = get_hdf(data_type=data_type)\n",
    "        df['item_rank'] = df.groupby(['buyer_admin_id', 'item_id']).cumcount(ascending=False) + 1\n",
    "        \n",
    "        all_user = df[['buyer_admin_id', 'is_train', 'country_id']].drop_duplicates()\n",
    "        all_user.to_hdf(path, 'all')\n",
    "\n",
    "        train = all_user[all_user['is_train']==1]\n",
    "        train.to_hdf(path, 'train')\n",
    "\n",
    "        test = all_user[all_user['is_train']==0]\n",
    "        test.to_hdf(path, 'test')\n",
    "\n",
    "        xx = all_user[all_user['country_id']=='xx']\n",
    "        xx.to_hdf(path, 'xx')\n",
    "\n",
    "        yy = all_user[all_user['country_id']=='yy']\n",
    "        yy.to_hdf(path, 'yy')\n",
    "\n",
    "        zz= all_user[all_user['country_id']=='zz']\n",
    "        zz.to_hdf(path, 'zz')\n",
    "\n",
    "        yyzz = all_user[all_user['country_id']!='xx']\n",
    "        yyzz.to_hdf(path, 'yyzz')\n",
    "        \n",
    "        buy = df[(df['irank']>1) & (df['buy_flag']==1)][['buyer_admin_id', 'is_train', 'country_id']].drop_duplicates()\n",
    "        train_buy = buy[buy['is_train']==1]\n",
    "        test_buy = buy[buy['is_train']==0]\n",
    "        \n",
    "        buy.to_hdf(path, 'buy')\n",
    "        train_buy.to_hdf(path, 'train_buy')\n",
    "        \n",
    "        buy = df[(df['irank']>1) & (df['buy_flag']==1)][['buyer_admin_id', 'is_train', 'country_id']].drop_duplicates()\n",
    "        train_buy = buy[buy['is_train']==1]\n",
    "        non_train_buy = train[~train['buyer_admin_id'].isin(train_buy['buyer_admin_id'])]\n",
    "\n",
    "        test_buy = buy[buy['is_train']==0]\n",
    "        non_test_buy = test[~test['buyer_admin_id'].isin(test_buy['buyer_admin_id'])]\n",
    "\n",
    "        buy.to_hdf(path, 'buy')\n",
    "        train_buy.to_hdf(path, 'train_buy')\n",
    "        non_train_buy.to_hdf(path, 'non_train_buy')\n",
    "        test_buy.to_hdf(path, 'test_buy')\n",
    "        non_test_buy.to_hdf(path, 'non_test_buy')\n",
    "        \n",
    "        rebuy = df[(df['irank']==1) & (df['item_rank']>1)][['buyer_admin_id', 'is_train', 'country_id']].drop_duplicates()\n",
    "        rebuy.to_hdf(path, 'rebuy')\n",
    "\n",
    "        cold = df[(df['irank']==1) & (df['item_rank']==1)][['buyer_admin_id', 'is_train', 'country_id']].drop_duplicates()\n",
    "        cold.to_hdf(path, 'cold')\n",
    "\n",
    "        cold_buy = cold[cold['buyer_admin_id'].isin(buy['buyer_admin_id'])]\n",
    "        cold_buy.to_hdf(path, 'cold_buy')\n",
    "        \n",
    "        non_cold_buy = cold[~cold['buyer_admin_id'].isin(buy['buyer_admin_id'])]\n",
    "        non_cold_buy.to_hdf(path, 'non_cold_buy')\n",
    "        \n",
    "        recall = pd.concat([cold ,test]).reset_index(drop=True)\n",
    "        recall.to_hdf(path, 'recall')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sample(dtype, data_type='base'):\n",
    "    \"\"\"样本选取\n",
    "    all(34751160): 用户交互所有样本(排除irank1)，用于在冷启动预测中，过滤用户已历史交互的样本 \n",
    "    rank(1410353): 用户buy_flag打标为1所有样本(buy_flag=1)，用于在rank模型中，irank=1为正样本, 其他为负样本\n",
    "    baseline(1899886): 按训练集irank去重后升序排序 \n",
    "    \n",
    "    Tips:baseline包含训练集历史有购买用户，rank仅包含训练集复购用户与测试集全量用户\n",
    "    \"\"\"\n",
    "    path = '../data/sample_' + data_type\n",
    "    if os.path.exists(path):\n",
    "        sample = pd.read_hdf(path, dtype).reset_index(drop=True)\n",
    "        return sample\n",
    "    else:\n",
    "        df = get_hdf('all', data_type)\n",
    "        rebuy_user = get_user('rebuy', data_type)\n",
    "        test_user = get_user('test', data_type)\n",
    "        cold_user = get_user('cold', data_type)\n",
    "\n",
    "        all_sample = df[df['irank']!=1][['buyer_admin_id', 'item_id', 'country_id', 'cate_id', 'store_id']].drop_duplicates()\n",
    "        all_sample.to_hdf(path, 'all')\n",
    "\n",
    "        rank_user = pd.concat([rebuy_user ,test_user])\n",
    "        rank_sample = df[(df['buy_flag']==1) & (df['buyer_admin_id'].isin(rank_user['buyer_admin_id']))]\\\n",
    "            [['buyer_admin_id', 'item_id', 'country_id', 'cate_id', 'store_id', 'is_train']].drop_duplicates()\n",
    "        irank1 = get_hdf('irank1', data_type)[['buyer_admin_id', 'item_id']].assign(irank=1)\n",
    "        rank_sample = rank_sample.merge(irank1, how='left', on=['buyer_admin_id', 'item_id'])\n",
    "        rank_sample['irank'] = rank_sample['irank'].fillna(0)\n",
    "        \n",
    "        baseline = df[(df['irank']>1) & (df['is_train']==1) & (df['buy_flag']==1)][['buyer_admin_id', 'item_id']].drop_duplicates().reset_index(drop=True)\n",
    "        baseline['irank'] = baseline.groupby(['buyer_admin_id']).cumcount() + 1\n",
    "        baseline.to_hdf(path, 'baseline')\n",
    "        \n",
    "        rank_sample = pd.merge(rank_sample, baseline.rename(columns = {'irank' : 'baseline'}), on=['buyer_admin_id', 'item_id'], how='left')\n",
    "        rank_sample.to_hdf(path, 'rank')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_prefix(df, exclude_columns, prefix):\n",
    "    if isinstance(exclude_columns, str):\n",
    "        exclude_columns = [exclude_columns]\n",
    "        \n",
    "    column_names = [col for col in df.columns if col not in exclude_columns]\n",
    "    df.rename(columns = dict(zip(column_names, [prefix + name for name in column_names])), inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_func(df, group_func_dic, group_key):\n",
    "    if isinstance(group_func_dic, str):\n",
    "        group_func_dic = [group_func_dic]\n",
    "        \n",
    "    features = df.groupby(group_key).agg(group_func_dic)\n",
    "    features.columns = [e[0] + \"_\" + e[1].upper() for e in features.columns.tolist()]\n",
    "    features.reset_index(inplace=True)\n",
    "    return features"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3 (PySpark)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
